<!DOCTYPE html>
<html>
<head>
    <title>Load data file</title>
    <meta http-equiv="Content-Script-Type" content="text/javascript">
    <style type="text/css">
        body { background: #eee; }
        p, th, td, label { font-family: sans-serif; font-size: 10pt}
        th { text-align: left; vertical-align: top }
        td { vertical-align: top; }
        .partition { padding: 0px }
        .hint { background-color: #DDD; color: #888; font-size: smaller }
        #data_file_record_count { font-size: small }
    </style>
    <!-- protovis -->
    <script type="text/javascript" src="MCAS/protovis-d3.2.js"></script>
    <!-- jquery -->
    <link type="text/css" href="jquery-ui-1.8.6.custom/css/smoothness/jquery-ui-1.8.6.custom.css" rel="stylesheet" />
    <script type="text/javascript" src="jquery/jquery-1.4.2.js"></script>
    <script type="text/javascript" src="jquery-ui-1.8.6.custom/js/jquery-ui-1.8.6.custom.min.js"></script>
    <!-- Google AJAX and visualization libraries -->
    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
    <script type="text/javascript">

google.load('visualization', '1', {packages:['table', 'corechart']});
google.setOnLoadCallback(googleLoaded);

function googleLoaded() {
    clearTimeout(googTime);
    // Enable file open button.
    $('#open_data_file')[0].disabled = false;
}
// Notify me if google.load fails.
var googTime = setTimeout('alert("The Google visualization library failed to load.")', 8000);

// Create view tabs.
var tabs;
$(function() { tabs = $('#view_control').tabs( {select: selectView} ).hide() } );

//
// Global variables
//
var fileReader;     // for W3C file API
var cols;           // info about data columns
var data;           // main data table (Google visualization API DataTable)
var col;            // map of column id's, eg, col.mcas_year usually equals 0
var filteredData;   // filtered data view (same column layout)

var summ = {                // summaries, to be zapped when filters change
    annual: null,           // datatable of annual summary values
    annualCol: null,        // column id map
    classGrade: null,       // datatable summarized by projected class graduation year and grade
    classGradeCol: null };  // column id map

var inputLength;    // total length of data file text
var lineMatch = /([^\r\n]*)[\r\n]+/g;  // match any combination of newline characters
var line;           // line buffer
var dataRows;       // data staged between filereader and datatable
var batchSize = 100;  // number of data rows to process between screen updates

var aypStateTarget = {  // state AYP CPI targets for ELA and math
year: [2002, 2004, 2006, 2008, 2010, 2012, 2014],	
10: [70.7, 75.6, 80.5, 85.4, 90.2, 95.1, 100],	
20: [53, 60.8, 68.7, 76.5, 84.3, 92.2, 100]
};

//
// Education data types, with inheritance
//
// [Future: Each object stores a series (array) of values.]
//
function EdDatum(value) {
    // Generic education data type.
    // The object returns its primitive value in 'value' or 'string' contexts.
    // Each ed data type has a prefered type (defaulting to 'string') and label (defaulting to 'ed data').
    // Ed data types can set a regular expression in validExp and numeric range constraints in validRange.
    // Ed data types can also set the validNull property to false to invalidate missing values (with the default validate method).
    // The default validate method uses any validExp, validRange, and validNull settings.
    // Ed data types can override the fix method to provide typical data fixups (when fix() is explicitly invoked).
    // The default fix method converts the value to the preferred type.
    this.value = value;
    this.prefer = 'string';  // keep this compatible with Google DataTable column types
    this.label = 'ed data';
    this.valueOf = function() {return this.value};
    this.toString = function() {return String(this.value)};
    this.validExp = null;  // override with regular expression if appropriate
    this.validRange = {min: null, max: null, integer: null};  // set min and/or max and/or boolean 'integer' to enable checking
    this.validNull = true;  // set false to invalidate missing values
    this.validate = edDatumValidate;
    this.fix = edDatumFix;
}

function edDatumValidate() {
    // Default validate method, to be run in context, via its call() method.
    //   E.g., edDatumValidate.call(this, v).
    // Validate regular expression, validNull, and any range parameters present.
    var v = this.value;
    if (v == null) return this.validNull;
    var match = this.validExp == null || this.validExp.test(v);
    var inRange = this.validRange == null 
        ||     (this.validRange.min == null || v >= this.validRange.min) 
        && (this.validRange.max == null || v <= this.validRange.max)
        && (this.validRange.integer == null || (v - Math.floor(v) == 0) == this.validRange.integer);
    return match && inRange;
}

function edDatumFix() {
    // Default fix method, to be run in context.
    // Return null --> not implemented; 0 --> value okay; 1 --> fixed; -1 --> could not fix.
    var wasValid = this.validate();
    switch (this.prefer) {
        case 'string':
            this.value = String(this.value);
            break;
        case 'number':
            this.value = Number(this.value);
            break;
    }
    if (this.validate())
        return wasValid ? 0 : 1;
    return -1;
}

function MAOrgCode(code) {
    // MA organization ID, a string of eight digits, often with leading zeros.
    // Values are required, by default.
    // The fix method will add leading zeros to a shorter number string.
    EdDatum.call(this, code);
    this.value = String(code);
    this.label = 'MA organization code';
    this.validExp = /^\d{8}$/;  // eight digits
    this.validNull = false;
    this.fix =
        function() {
            var digitsExp = /^\d+$/;
            if (this.validate()) return 0;
            this.value = String(this.value);
            if (digitsExp.test(this.value) && this.value.length < 8) {
                // Prepend zeros.
                while (this.value.length < 8)
                    this.value = '0' + this.value;
                return 1;
            }
            return -1;
        };
}
MAOrgCode.prototype = new EdDatum;

function StudentID(id) {
    // Student ID, stored as a string.
    // Value required.
    EdDatum.call(this, id);
    this.value = String(id);
    this.label = 'student ID';
    this.validNull = false;
}
StudentID.prototype = new EdDatum;

function MASASID(sasid) {
    // MA SASID, state assigned student identifier, 10 digits.
    // Value required.
    StudentID.call(this, sasid);
    this.label = 'SASID';
    this.validExp = /^\d{10}$/;  // ten digits
}
MASASID.prototype = new StudentID;

function Name(value) {
    // A person's name.
    EdDatum.call(this, value);
    this.label = 'name';
}
Category.prototype = new EdDatum;

function Teacher(value) {
    Name.call(this, value);
    this.label = 'teacher';
}
Teacher.prototype = new Name;

function CalendarYear(year) {
    // As opposed to SchoolYear.
    // Valid as an integer between 1900 and 3000.
    // Values are required, by default.
    // The fix method will add 2000 to values less than 100.
    // The asDate method returns a Javascript date of January 1 in the given year.
    EdDatum.call(this, year);
    this.value = numberOrMissing(year);
    this.prefer = 'number';
    this.label = 'calendar year';
    this.validRange.min = 1900;
    this.validRange.max = 3000;
    this.validRange.integer = true;
    this.validNull = false;
    this.fix = 
        function() { 
            if (this.value < 100 && this.validate(this.value + 2000)) { 
                this.value += 2000;
                return 1;
            }
            return this.validate() ? 0 : -1;
        };
    this.asDate = 
        function(mo, da) {
            !mo && (mo = 1);
            !da && (da = 1);
            return new Date(this.value, mo, da);
        };
}
CalendarYear.prototype = new EdDatum;

function SchoolYear(year) {
    // School years, like financial years, are denoted by their ending year.
    // For example, the 2010-2011 school year is denoted by 2011.
    // The fix method attempts to convert strings like '2010-11' to school years.
    // Use the commonFormat method to generate the school year as yyyy-yyyy.
    CalendarYear.call(this, year);
    this.value = year;
    this.label = 'school ending year';
    this.fix =
        function() {
            if (this.value.length > 4) {
                var commonExp = /(\d{2,4})\s*(-|\/| )\s*(\d{2,4})/;
                var match = commonExp.exec(this.value);
                if (match) {
                    var year1 = Number(match[1]);
                    var year2 = Number(match[3]);
                    if (year1 < 100) year1 += 2000;
                    if (year2 < 100) year2 += 2000;
                    if (year2 = year1 + 1) {
                        this.value = year2;
                        return 1;
                    }
                }
            }
            if (this.value < 100 && this.validate(this.value + 2000)) { 
                this.value += 2000;
                return 1;
            }
            return this.validate() ? 0 : -1;
        };
    this.commonFormat =
        function () {
            if (this.validate())
                return (this.value - 1) + '-' + this.value;
            else
                return this.value;
        };
}
SchoolYear.prototype = new CalendarYear;

function Grade(grade) {
    // The validate method looks for an integer of one or two digits or one of the strings K, PK, or SP.
    // Missing values are, by default, invalid.
    // The fix method tries uppercasing the value. For valid values, fix will strip a leading zero.
    //   If prefer = 'string' fix will add leading zeros to single-digit grades.
    //   If prefer = 'number' fix will strip leading zeros and convert alphabetic designators to numbers.
    // By default, prefer = 'number'.
    // Use the asNumber method to return a grade number, 0 for K, -1 for PK, and 99 for SP.
    // Use the asString method (not the toString method) to add a leading zero to single-digit grade numbers.
    EdDatum.call(this, grade);
    this.value = String(grade);
    this.prefer = 'number';
    this.label = 'grade';
    this.validExp = /^\d{1,2}$|^K$|^PK$|^SP$/;
    this.validNull = false;
    this.asNumber = 
        function (v) {
            if (v === undefined) v = this.value;
            var intExp = /^\d+$/;
            if (intExp.test(v))
                return Number(v);
            else switch (v.toUpperCase()) {
                case 'K':
                    return 0;
                case 'PK':
                    return 0;
                case 'SP':
                    return 99;
                default:
                    return NaN;
            }
        };
    this.asString =
        function (v) {
            if (v === undefined) v = this.value;
            var intExp = /^\d+$/;
            if (intExp.test(v))
                return v.length == 1 ? '0' + String(v) : String(v);
            else
                return String(v);
        };
    this.fix =
        function() {
            var oldVal = this.value;
            if (this.prefer == 'string') {
                this.value = this.value.toUpperCase();
                this.value = this.asString();
                if (this.validate())
                    return this.value == oldVal ? 0 : 1;
                else
                    this.value = oldVal;
            } else {  // 'number'
                if (this.validate()) {
                    this.value = this.asNumber();
                    return this.value == oldVal ? 0 : 1;
                }
                return -1;  // couldn't fix it
            }
        };
}
Grade.prototype = new EdDatum;

var MCASSubjectName = {
    // MCAS subject names indexed by subject code.
    // Change the values here to control how subject names appear.
    10: 'English',
    20: 'math',
    30: 'science',
    31: 'biology',
    32: 'chemistry',
    33: 'physics',
    34: 'technology/engineering',
};

function MCASSubjectCode(code) {
    // MCAS test subject codes include 10, 20, and 30-33 at present.
    // Missing values are, by default, invalid.
    // The constructor accepts numeric codes.
    // It will attempt to interpret subject names 
    //   and store the corresponding code *at construction time*.
    // Prefer this storage format for an MCAS subject category over storing the subject name directly.
    // Use MCASSubjectName[code] to retrieve subject name strings.
    EdDatum.call(this, code);
    this.prefer = 'number';
    this.label = 'MCAS subject code';
    this.formatter = new google.visualization.NumberFormat( { fractionDigits: 1 } );
    this.validNull = false;
    this.validate =
        function() {
            if (this.code == null && this.validNull) return true;
            switch (this.code) {
                case 10:
                case 20:
                case 30:
                case 31:
                case 32:
                case 33:
                    return true;
                default:
                    return false;
            }
        };
    if (!this.validate()) {
        if (/english|language|ela|reading/i.test(code))
            this.value = 10;
        else if (/math/i.test(code))
            this.value = 20;
        else if (/science/i.test(code))
            this.value = 30;
        else if (/bio/i.test(code))
            this.value = 31;
        else if (/chem/i.test(code))
            this.value = 32;
        else if (/physic/i.test(code))
            this.value = 33;
        else if (/tech/i.test(code))
            this.value = 34;
        // Otherwise keep whatever value was supplied.
    }
}
MCASSubjectCode.prototype = new EdDatum;

function Category(value) {
    // A generic category type.
    // Missing values are valid, by default.
    EdDatum.call(this, value);
    this.label = 'category';
}
Category.prototype = new EdDatum;

function BooleanCategory(value) {
    // A yes/no category, which can be supplied as 1/0, true/false, etc.
    // After running the fix method, the value will be Y, N, or missing, unless the fix fails.
    // To interpret missing values as no's, set the property missingMeansNo to true and run the fix method.
    Category.call(this, value);
    this.label = 'yes/no';
    this.validExp = /^y|^n|^1|^0|^t|^f/i;
    this.missingMeansNo = false;
    this.fix =
        function() {
            var normalExp = /^Y$|^N$/;
            var yesExp = /^y|^1|^t/i;
            var noExp = /^n|^0|^f/i;
            if (normalExp.test(this.value)) return 0;  // no fix needed
            if (this.value == null) {
                if (this.missingMeansNo) {
                    this.value = 'N';
                    return 1;  // fixed it
                }
                if (this.validNull) 
                    return 0;  // okay as missing value
                return -1;  // couldn't fix it
            }
            if (yesExp.test(this.value)) {
                this.value = 'Y';
                return 1;  // fixed it
            }
            if (noExp.test(this.value)) {
                this.value = 'N';
                return 1;  // fixed it
            }
            return -1;  // couldn't fix it
        };
}
BooleanCategory.prototype = new Category;

function Rating(value) {
    // A text rating.
    // Override the order method to assign ordinal numbers to rating values.
    EdDatum.call(this, value);
    this.value = String(value);
    this.prefer = 'string';
    this.label = 'rating';
    this.order = function() {return null};
}
Rating.prototype = new EdDatum;

function Score(value) {
    // A numeric score.
    EdDatum.call(this, value);
    this.value = numberOrMissing(value);
    this.prefer = 'number';
    this.label = 'score';
}
Score.prototype = new EdDatum;

function PointScore(value) {
    // A non-negative, integral score.
    Score.call(this, value);
    this.validRange.min = 0;
    this.validRange.integer = true;
}
PointScore.prototype = new Score;

function DIBELSScore(points) {
    PointScore.call(this, points);
    this.label = 'DIBELS score';
}
DIBELSScore.prototype = new PointScore;

function SRAScore(points) {
    // Fractional scores.
    Score.call(this, points);
    this.label = 'SRA score';
    this.validRange.min = 0;
}
SRAScore.prototype = new Score;

function MCASRawScore(points) {
    // MCAS raw scores for items, tests, aggregate totals.
    PointScore.call(this, points);
    this.label = 'raw score';
}
MCASRawScore.prototype = new PointScore;

function MCASMaximumScore(points) {
    // Maximum points possible.
    MCASRawScore.call(this, points);
    this.label = 'maximum score possible';
}
MCASMaximumScore.prototype = new MCASRawScore;

function MCASAverageScore(points) {
    // Average of raw point scores, may be fractional.
    Score.call(this, points);
    this.label = 'average raw score';
}
MCASAverageScore.prototype = new Score;

function MCASScaledScore(score) {
    // MCAS scaled scores, even values between 200 and 280.
    PointScore.call(this, score);
    this.label = 'scaled score';
    this.validate =
        function() {
            return this.value >= 200 && this.value <= 280 && this.value % 2 == 0;
        };
}
MCASScaledScore.prototype = new PointScore;

function MCASStudentCPI(cpi) {
    // MCAS CPI, comprehensive performance index.
    // This is the individual student value, which can be 0, 25, 50, 75, or 100, or null.
    PointScore.call(this, cpi);
    this.label = 'CPI';
    this.validate =
        function() {
            if (this.value == null) return true;
            switch (this.value) {
                case 0:
                case 25:
                case 50:
                case 75:
                case 100:
                    return true;
                default:
                    return false;
            }
        };
}
MCASStudentCPI.prototype = new PointScore;

var MCASPerformanceLevels = {
    'WF': { long: 'Warning/Failing',   order: 1,   match: /^wf$|^warn\w*.fail/i   },
    'W':  { long: 'Warning',           order: 1.1, match: /^w$|^warn/i            },
    'F':  { long: 'Failing',           order: 0.9, match: /^f$|^fail/i            },
    'NI': { long: 'Needs Improvement', order: 2,   match: /^ni$|^needs/i          },
    'P':  { long: 'Proficient',        order: 3,   match: /^p$|^prof/i            },
    'A':  { long: 'Advanced',          order: 4,   match: /^a$|^adv/i             },
    'P+': { long: 'Beyond Proficient', order: 4.1, match: /^p\+$|^prof\w* (\+|plus)|^beyond prof/i }
};

function MCASPerformanceLevelRecognizer(s) {
    // Attempt to recognize MCAS performance levels.
    // Return the short name if found, otherwise return null.
    if (s == null) return null;
    for (var i in MCASPerformanceLevels)
        if (MCASPerformanceLevels[i].match.test(s))
            return i;
    return null;
}

function MCASPerformanceLevel(value) {
    // MCAS performance levels are Failing, Warning, Needs Improvement, Proficient, Beyond Proficient, and Advanced.
    // Some abbreviations: F, W, WF, NI, P, P+, A.
    // MCAS ALT performance levels include INP, PRG.
    // Other codes slipped into performance levels are ABS, MED, ELL, PAS,...
    Rating.call(this, value);
    this.order = 
        function() {
            var quick = MCASPerformanceLevels[this.value];
            if (quick) return quick.order;
            var shortName = MCASPerformanceLevelRecognizer(this.value);
            if (shortName == null) 
                return 99;
            else
                return MCASPerformanceLevels[shortName].order;
        };
}
MCASPerformanceLevel.prototype = new Rating;

function MCASSGP(value) {
    // Student growth percentiles.
    Score.call(this, value);
    this.label = 'student growth percentile';
    this.validRange.min = 1;
    this.validRange.max = 99;
}
MCASSGP.prototype = new Score;

//
// Functions to read data
//
function readFile() {
    // Read data from the selected file.
    // Clear display and data status.
    removeControls();
    data = new google.visualization.DataTable();  // fresh datatable
    // Use the W3C file API.
    var files = $('[name=input_data_file]')[0].files;
    if (!files)
        alert("This tool relies on new web technologies not supported by the web browser you are using. We recommend using Firefox version 3.6 or newer.");
    else if (files.length) {
        var file = files[0];
        if (file.type == 'text/plain') {
            fileReader = new FileReader();  // use the W3C file API
            fileReader.onerror = loadError;
            fileReader.onload = parseText;
            fileReader.readAsText(file);
        } else {
            alert('This program only understands plain text files. The file you selected is of ' + ( file.type ? file.type : 'unknown' ) + ' type.');
        }
    }
}

function loadError() {
    alert('An error occurred reading the data file.');
}

function parseText() {
    // First create a matrix of text strings from the delimited file input.
    dataRows = new Array();  // two dimensional array corresponding to file content
    inputLength = fileReader.result.length;
    progressbar(0);
    // Extract column names from first line.
    line = lineMatch.exec(fileReader.result);
    if (line) {
        analyzeColumns(line[1].split('\t'));
        if (lineMatch.lastIndex) {  // more input
            line = lineMatch.exec(fileReader.result);  // set up next line
            setTimeout('parseContinue()', 10);  // continue parsing after refreshing screen
        }
    }
}

var dataTypeProductions = [
    [/years/i, Category],
    [/(sims)|(school).*year/i, SchoolYear],
    [/year/i, CalendarYear],
    [/teacher/i, Teacher],
    [/dibels/i, DIBELSScore],
    [/sra/i, SRAScore],
    [/grade/i, Grade],
    [/oct.?1/i, BooleanCategory],
    [/sasid/i, MASASID],
    [/status/i, Category],
    [/race/i, Category],
    [/gender/i, Category],
    [/alt.*assess/i, BooleanCategory],
    [/lep|limited english/i, BooleanCategory],
    [/income|frpl|reduced/i, BooleanCategory],
    [/504/i, BooleanCategory],
    [/educat.*environ/i, Category],
    [/(sped)|(special.ed).*eval/i, Category],
    [/sped|special.ed|disab/i, BooleanCategory],
    [/title.(i|1)/i, BooleanCategory],
    [/(dist\a*)|(school).?(id)|(code)/i, MAOrgCode],
    [/subj/i, MCASSubjectCode],
    [/max.*score/i, MCASMaximumScore],
    [/raw.?score/i, MCASRawScore],
    [/sgp|growth.perc/i, MCASSGP],
    [/cpi/i, MCASStudentCPI],
    [/scaled.?score|\bs.?s\b/i, MCASScaledScore],
    [/perf.*lev.*/i, MCASPerformanceLevel],
    [/score|point/i, Score],
    [/(first|last).*name/i, Name],
    [/stu.*name/i, Name],
    [/name/i, Name],
    [/.*/, EdDatum]  // guaranteed match
];

function classifyDataName(n) {
    for (var i in dataTypeProductions)
        if (dataTypeProductions[i][0].test(n))
            return dataTypeProductions[i][1];
}

function analyzeColumns(names) {
    // Analyze an array of column names, store results in cols, and add columns to the data table.
    var nameReplace = /\W+/g;  // replace matching characters with underscores
    var nameTrim = /^_+|_+$/g;  // trim underscores
    var primitiveType;  // 'string', 'number'
    cols = new Array(names.length);  // column info, each entry a map
    var i;  // index through names array
    for (i = 0; i < names.length; i++) {
        cols[i] = new Object();
        cols[i].label = names[i];
        cols[i].id = names[i].trim().toLowerCase().replace(nameReplace, '_').replace(nameTrim, '');
        cols[i].type = classifyDataName(names[i]);  // ed data type (not primitive type)
        cols[i].eg = new cols[i].type();  // reference example of type
        // Add a column to the main data table.
        data.addColumn(cols[i].eg.prefer, cols[i].label, cols[i].id);
    }
    addCalcColumns();
}

function addCalcColumns() {
    // Add percent scores.
    var rawCol, maxCol;
    for (c in cols) {
        if (cols[c].type == MCASRawScore) {
            rawCol = c;
        }
        if (cols[c].type == MCASMaximumScore) {
            maxCol = c;
        }
    }
    if (rawCol && maxCol) {
        cols.push(
            {
                label: 'MCAS Percent Score',
                id: 'mcas_percent_score',
                type: Score,
                eg: new Score(),
                calculated: true
            }
        );
        // Add a column to the main data table.
        var i = cols.length - 1;
        data.addColumn(cols[i].eg.prefer, cols[i].label, cols[i].id);
    }
}

function parseContinue() {
    // Parse batches of input lines, refreshing the screen in between.
    var batch = batchSize;
    while (line && line.index < inputLength && batch--) {
        dataRows.push(line[1].split('\t'));         // store fields from one line
        line = lineMatch.exec(fileReader.result);   // match next line
    }
    // Update progress bar.
    progressbar(lineMatch.lastIndex ? (lineMatch.lastIndex / inputLength * 100) : 100);
    if (line != null && lineMatch.lastIndex < inputLength) {  // more input
        setTimeout('parseContinue()', 10);  // allow a screen refresh before continuing
    } else {  // no more input
        fileReader = null;  // release input file data
        fixData();
        //convertNumbers();  // convert numeric columns
        data.addRows(dataRows);  // add all rows to main data table
        dataRows = null;  // release input data array
        progressbar(-1);  // remove progress bar
        // Create a column id map for the main data table.
        col = new Object();
        for (var iCol = 0; iCol < data.getNumberOfColumns(); iCol++) {
            col[data.getColumnId(iCol)] = iCol;
        }
        showControls();
        $('#data_file_record_count').html(data.getNumberOfRows() + ' records');
        // Process possibly empty filters and create filteredData view.
        filterData();  
        tabs.tabs('select', 0);
        displayColumnSummary(filteredData);
    }
}

function fixData() {
    // Use ed data types to fix up and adapt to data.
    var c;  // column index
    var nCols = cols.length;
    var r;  // row index
    var nRows = dataRows.length;
    var temp;  // temporary ed data object
    var fixCode;
    for (c = 0; c < nCols; c++) {
        cols[c].fixResults = [0, 0, 0];  // counts of [0] failed, [1] unnecessary, and [2] successful fixes
        for (r = 0; r < nRows; r++) {
            temp = new (cols[c].type)(dataRows[r][c]);
            fixCode = temp.fix();
            if (fixCode != null)
                cols[c].fixResults[fixCode + 1]++;
            dataRows[r][c] = temp.value;
        }
    }
}

function convertNumbers() {
    for (var col = 0; col < cols.length; col++)
        if ((new cols[col].type).prefer == 'number')
            for (var row = 0; row < dataRows.length; row++) {
                var n = parseInt(dataRows[row][col]);
                if (!isNaN(n))
                    dataRows[row][col] = n;
                else
                    dataRows[row][col] = null;
            }
}

//
// Utility functions
//
function progressbar(n) {
    if (n < 0) {
        $( "#data_file_progress" ).progressbar('destroy');
    } else {
        $( "#data_file_progress" ).progressbar({ value: n });
    }
}

function safe(s) {
    // Return an HTML-safe string with character entities.
    s = s.replace(/&/g, '&amp;');
    s = s.replace(/</g, '&lt;');
    s = s.replace(/>/g, '&gt;');
    return s;
}

function numberOrMissing(v) {
    // Return a number, except return null for an empty string or "NA".
    if (String(v) == '' || String(v).toUpperCase() == 'NA')
        return null;
    else
        return Number(v);
}

function denoteEmpty(s) {
    // Denote an empty string by <blank>.
    return s == '' ? '<blank>' : s;
}

//
// UI functions
//
function newFile() {
    // User selected a new file (but hasn't opened it yet).
    // Clear display and data status.
    removeControls();
    // Focus the Open button.
    $('#open_data_file').focus();
}

function removeControls() {
    // Remove or hide controls preparatory to reading new data.
    $('#data_file_record_count').empty();
    $('#filter_area').empty();
    $('#filter_control').css('visibility', 'hidden');
    $('div#view_control > div').empty();
}

function showControls() {
    displayCategories();
    $('#view_control').show();
    //displayMetrics();
}

var filterControl;  // map of filter HTML strings
var filterHide = ['SASID'];  // array of labels for hidden filters

function displayCategories() {
    var label;  // column label
    var tr;  // HTML table row element
    $("#filter_area").empty();
    $("#filter_control").css('visibility', 'visible');
    // Create menus.
    filterControl = createFilters();

    $('#filter_area').append('<table><tr></tr><tr></tr></table>');
    tr = $('#filter_area tr');  // two rows
    for (label in filterControl)
        if (! filterHide.some(function (l) { return l == label })) {
            $(tr[0]).append('<td style="vertical-align:bottom">' + label + '</td>');
            $(tr[1]).append('<td style="vertical-align:top">' + filterControl[label] + '</td>');
        }
}

function createFilters() {
    var c;  // column index
    var filter = new Object();  // map of filter HTML strings, by column label
    for (c = 0; c < cols.length; c++) {
        switch (cols[c].type) {
            case MAOrgCode:
            case CalendarYear:
            case Grade:
            case MCASSubjectCode:
            case Category:
            case BooleanCategory:
                filter[data.getColumnLabel(c)] = categoryFilter(c, data);
                break;
        }
    }
    return filter;
}

function categoryFilter(c, t) {
    // Create an HTML menu (SELECT element) for the values in column c of table or view t.
    // Return the menu as HTML text.
    var values;         // distinct values
    var i;              // value index
    var options = '';   // HTML text for menu options
    var html;           // HTML text for complete filter control
    values = t.getDistinctValues(c);
    for (i in values) {
        options += '<option>' + values[i] + '</option>';
    }
    html = '<select multiple size=4 class="menu" name="' + t.getColumnId(c) + '" onchange="filterData()">' + options + '</select>';
    return html;
}

function filterSelection(type, id) {
    // Find a filter of the specified type and optionally id.
    // Return null if no filter qualifies.
    // Otherwise return a list of selected values, which may be empty.
    var found = false;
    var values = new Array();
    $("select.menu").each(
            function() {
                var thisId = this.name;  // menu name is the corresponding column id
                var thisCol = col[thisId];  // column index
                if (cols[thisCol].type == type) {
                    if (id === undefined || id == thisId) {  // this menu matches
                        found = true;
                        $('option[selected]', this).each(function() { values.push(this.value) });
                    }
                }
            });
    return found ? values : null;
}

var metrics = { scores: [], rankscores: [], ratings: [] };

function displayMetrics() {
    // Determine and display controls for available metrics.
    var c;  // column index
    var m;  // metric index (in metrics)
    var row;  // HTML table row index
    var rowHTML;
    var html;
    for (c = 0; c < cols.length; c++) {
        switch (cols[c].type) {
            case Score:
            case PointScore:
            case MCASRawScore:
            case MCASAverageScore:
            case MCASStudentCPI:
            case MCASMaximumScore:
                metrics.scores.push(c);
                break;
            case MCASScaledScore:
            case MCASSGP:
                metrics.rankscores.push(c);
                break;
            case Rating:
            case MCASPerformanceLevel:
                metrics.ratings.push(c);
                break;
        }
    }
    html = '';
    for (row = 0; ; row++) {
        rowHTML = '';
        if (metrics.scores.length > row) {
            rowHTML += '<td><input name="' + cols[metrics.scores[row]].id + '" type="checkbox">'
                + safe(cols[metrics.scores[row]].label);
        } else {
            rowHTML += '<td>';
        }
        if (metrics.rankscores.length > row) {
            rowHTML += '<td><input name="' + cols[metrics.rankscores[row]].id + '" type="checkbox">'
                + safe(cols[metrics.rankscores[row]].label);
        } else {
            rowHTML += '<td>';
        }
        if (metrics.ratings.length > row) {
            rowHTML += '<td><input name="' + cols[metrics.ratings[row]].id + '" type="checkbox">'
                + safe(cols[metrics.ratings[row]].label);
        } else {
            rowHTML += '<td>';
        }
        if (rowHTML.indexOf('input') != -1)
            html += '<tr>' + rowHTML + '</tr>';
        else
            break;
    }
    html = '<form><table>' + html + '</table></form>';
    $('#metric_control').html(html);
}

function selectView(event, ui) {
    // Handle user selection of a new data view/tab.
    switch (ui.index) {
        case 0:
            displayColumnSummary(filteredData);
            break;
        case 1:
            displayTable();
            break;
        case 2:
            displayAnnualSummaryTable();
            break;
        case 3:
            displayTimeSeries();
            break;
        case 4:
            displayAYP();
            break;
        case 5:
            displayGradesAndCohorts();
            break;
    }
}

//
// Data manipulation functions
//
function filterData() {
    // Filter rows from the main data table into a data view.
    filteredData = new google.visualization.DataView(data);
    // Clear summary tables.
    for (var t in summ) summ[t] = null;
    // If we encounter a menu selection, make rows an array
    // and accumulate matching row indices.
    var rows = null;  // no seletions yet
    // Check each HTML menu for selections.
    $("select.menu").each(
            function() {
                if ($('option[selected]', this).length) {
                    var matchingThis = matchingMenu(this);
                    if (rows == null)  // first selection encountered
                        rows = matchingThis.sort(numOrder);
                    else {
                        // Keep only rows that match in each menu where there are selections.
                        rows = rows.concat(matchingThis);
                        rows.sort(numOrder);
                        rows = rows.filter(duplicate);
                    }
                }
            });
    // Set rows included in the filtered view.
    if (rows != null)  // at least one menu had a selection
        filteredData.setRows(rows);
    $('#filter_count').html(filteredData.getNumberOfRows());
}

function matchingMenu(menu) {
    // If there are no selections in this menu return null.
    // Otherwise return an array of matching row indices.
    // The array may be empty, [].
    var id = menu.name;  // menu name is the same as the corresponding column id
    var i;  // option index
    var nOptions = menu.options.length;
    var val;  // selected value
    var matches = null;  // no selections encountered yet
    for (i = 0; i < nOptions; i++) {
        if (menu.options[i].selected) {
            val = menu.options[i].value;  // text value
            if (data.getColumnType(col[id]) == 'number') {
                val = Number(val);  // numeric value
            }
            if (matches == null)  // first selection in this menu
                matches = data.getFilteredRows([{column: col[id], value: val}]);
            else  // add additional matches
                matches = matches.concat(data.getFilteredRows([{column: col[id], value: val}]));
        }
    }
    return matches;
}

function unique(val, i, array) { 
    // True for the first occurrence of a value, when iterating through a sorted array.
    return i == 0 || val != array[i-1];
}

function duplicate(val, i, array) { 
    // True for subsequent occurrences of a value, when iterating through a sorted array.
    return i > 0 && val == array[i-1];
}

function numOrder(a, b) { return a - b; }  // for numeric sorting

function summarizeAnnual() {
    // Create a summary table by test year, using filtered data.
    var oneDigitAfterDecimal = new google.visualization.NumberFormat( { fractionDigits: 1 } );
    if (summ.annual) return;  // already in place
    summ.annual = google.visualization.data.group(filteredData, 
            [{column: col.mcas_year, modifier: function (y) {return String(y)}, type: 'string', label: 'year', id: 'year'}], 
            [{column: col.mcas_sasid, aggregation: countDistinct, type: 'number', label: 'students', id: 'students'},
             {column: col.mcas_student_cpi, aggregation: google.visualization.data.avg, type: 'number', label: 'CPI', id: 'cpi'},
             {column: col.mcas_student_performance_level_clean, aggregation: countAll, type: 'number', label: 'performance levels reported', id: 'performance_level_count'},
             {column: col.mcas_student_performance_level_clean, aggregation: countA, type: 'number', label: 'A/P+', id: 'A'},
             {column: col.mcas_student_performance_level_clean, aggregation: countP, type: 'number', label: 'P', id: 'P'},
             {column: col.mcas_student_performance_level_clean, aggregation: countNI, type: 'number', label: 'NI', id: 'NI'},
             {column: col.mcas_student_performance_level_clean, aggregation: countWF, type: 'number', label: 'W/F', id: 'WF'} ]);
    // Create a column id map.
    summ.annualCol = new Object();
    for (var iCol = 0; iCol < summ.annual.getNumberOfColumns(); iCol++) {
        summ.annualCol[summ.annual.getColumnId(iCol)] = iCol;
    }
    // Format columns.
    oneDigitAfterDecimal.format(summ.annual, summ.annualCol.cpi);
}

function summarizeClassGrade() {
    // Create a summary table by projected class graduation year and grade.
    if (summ.classGrade) return;  // already in place
    function identity(v) { return v }
    summ.classGrade = google.visualization.data.group(filteredData,
            [ {column: col.mcas_grade, modifier: identity, type: 'number', label: 'grade', id: 'grade' },
              {column: col.mcas_year, modifier: identity, type: 'number', label: 'year', id: 'year' }],
            [ {column: col.mcas_student_performance_level_clean, aggregation: countAll, type: 'number', label: 'performance levels reported', id: 'performance_level_count'},
              {column: col.mcas_student_performance_level_clean, aggregation: countA, type: 'number', label: 'A/P+', id: 'A'},
              {column: col.mcas_student_performance_level_clean, aggregation: countP, type: 'number', label: 'P', id: 'P'},
              {column: col.mcas_student_performance_level_clean, aggregation: countNI, type: 'number', label: 'NI', id: 'NI'},
              {column: col.mcas_student_performance_level_clean, aggregation: countWF, type: 'number', label: 'W/F', id: 'WF'},
              {column: col.mcas_sasid, aggregation: countDistinct, type: 'number', label: 'students', id: 'students'} ]);
    summ.classGrade.set
    // Add projected class graduation years.
    summ.classGrade.insertColumn(0, 'number', 'class of', 'classOf');
    for (row = 0; row < summ.classGrade.getNumberOfRows(); row++) {
        // Projected graduation year = school ending year + 12 - grade.
        // (We're assuming Spring tests so test year = school ending year.)
        summ.classGrade.setValue(row, 0, summ.classGrade.getValue(row, 2) + 12 - summ.classGrade.getValue(row, 1));
    }
    // Create a column id map.
    summ.classGradeCol = new Object();
    for (var iCol = 0; iCol < summ.classGrade.getNumberOfColumns(); iCol++) {
        summ.classGradeCol[summ.classGrade.getColumnId(iCol)] = iCol;
    }
    // Sort by grade and year.
    summ.classGrade.sort(summ.classGradeCol.grade, summ.classGradeCol.year);
}

//
// Auxiliary functions to count values.
//
function countAll(values) {
    return values.filter(function (v) {return v.length}).length;
}
function countDistinct(values) {
    return pv.uniq(values.filter(function (v) {return v.length})).length
}
//
// Auxiliary functions to count performance level occurences.
//
function countA(levels) {
    return levels.filter(function (v) {return v == 'A' || v == 'P+'}).length;
}
function countP(levels) {
    return levels.filter(function (v) {return v == 'P'}).length;
}
function countNI(levels) {
    return levels.filter(function (v) {return v == 'NI'}).length;
}
function countWF(levels) {
    return levels.filter(function (v) {return v == 'WF'}).length;
}

//
// Visualization functions
//
function displayColumnSummary(t) {
    // Summarize data columns in a datatable or dataview.
    var displayRow; // HTML table row
    var range;      // range of values in a column
    var text;       // text to insert

    $("#column_view").empty();
    $("#column_view").append('<table id="tbl_summary"></table>');
    // Display a row of information for each table column.
    for (c = 0; c < t.getNumberOfColumns(); c++) {
        $('#tbl_summary').append('<tr><td>' + t.getColumnLabel(c) + '</td></tr>');
        displayRow = $('#tbl_summary tr:last');
        // Display variable type.
        displayRow.append('<td>' + cols[c].eg.label + '</td>');
        // Display range of values.
        range = t.getColumnRange(c);
        if (range.min == null)
            text = 'empty';
        else
            text = safe(denoteEmpty(range.min) + ' - ' + denoteEmpty(range.max));
        displayRow.append('<td>' + text + '</td>');
    }
}

function displayTable() {
    // Display data in a table.
    $("#table_view").empty();
    var table = new google.visualization.Table(document.getElementById('table_view'));
    table.draw(filteredData, {showRowNumber: true, page: 'enable', pageSize: 100});
}

function displayAnnualSummaryTable() {
    // Display table of annual summary data.
    $("#annual_summary_view").empty();
    summarizeAnnual();
    var table = new google.visualization.Table(document.getElementById('annual_summary_view'));
    table.draw(summ.annual, {showRowNumber: true, page: 'enable', pageSize: 100, allowHTML: true});
}

var vis;  // global for debugging
function displayTimeSeries() {
    // Display time series plots.
    $('#performance_view').empty();
    summarizeAnnual();
    var wOverall = screen.availWidth * 0.9, 
        hOverall = screen.availHeight * 0.75;
    $('#performance_view').width(wOverall);
    $('#performance_view').height(hOverall);
    // Dimension side-by-side plots.
    var pad = {left: 30, top: 40, right: 90, bottom: 20};  // pad around plots
    var wMax = 600, hMax = 400;  // keep sizes in reign
    // Plot panel sizes (not including labels that overflow panel):
    var w = Math.min( (wOverall / 2) - pad.left - pad.right, wMax ), 
        h = Math.min( hOverall - pad.top - pad.bottom, hMax );
    // Plot area sizes (including padding around plot panels):
    var wFrame = w + pad.left + pad.right,
        hFrame = h + pad.top + pad.bottom;
    // Scales:
    var years = summ.annual.getDistinctValues(0);
    var yCPI = pv.Scale.linear(0, 100).range(0, h);
    var yStudents = pv.Scale.linear(0, summ.annual.getColumnRange(summ.annualCol.students).max).range(0, h);
    var x = pv.Scale.ordinal(years).splitFlush(0, w);
    // Panels:
    var lh, rh, perfLevPanel;
    vis = new pv.Panel()
        .canvas('performance_view')  // root canvas
        .width(wFrame * 2)
        .height(hFrame)
        ;
    lh = vis.add(pv.Panel)  // left-hand panel
        .width(w)
        .height(h)
        .left(pad.left)
        .top(pad.top)
        .fillStyle('#ccc')
        ;
    perfLevPanel = lh.add(pv.Panel)  // performance level panel
        ;
    addPerformanceLevelsSeries(perfLevPanel, x, h, normalized=true);
    lh.add(pv.Rule)  // horizontal = years
        .data(years)
        .left(x)
        .strokeStyle('#eee')
        .anchor('bottom')
        .add(pv.Label)
        .text(function(y) { return y } )
        .font('12px sans-serif')
        ;
    lh.add(pv.Rule)  // CPI
        .data(yCPI.ticks())
        .visible(function() { return this.index > 0 })
        .bottom(yCPI)
        .width(10)
        .strokeStyle('#eee')
        .anchor('left')
        .add(pv.Label)
        .text(function(cpi) { return cpi } )
        .font('12px sans-serif')
        .parent
        .add(pv.Line)  // plot CPI
        .data(years)
        .strokeStyle('#444')
        .left(x)
        .bottom(function(y) { return yCPI(summ.annual.getValue(this.index, summ.annualCol.cpi)) } )
        .add(pv.Dot)
        .fillStyle(function() { return this.strokeStyle() } )
        .title(function(y) { return String(y) + ' CPI: ' + summ.annual.getValue(this.index, summ.annualCol.cpi).toFixed() } )  // popup tip
        ;
    rh = vis.add(pv.Panel)  // right-hand panel
        .height(h)
        .width(w)
        .left(wFrame + pad.left)
        .top(pad.top)
        .fillStyle('#ccc')
        ;
    rh.add(pv.Rule)  // horizontal = years
        .data(years)
        .left(x)
        .strokeStyle('#eee')
        .anchor('bottom')
        .add(pv.Label)
        .text(function(y) { return y } )
        .font('12px sans-serif')
        ;
    rh.add(pv.Rule)  // students tested
        .data(yStudents.ticks())
        .bottom(yStudents)
        .strokeStyle('#eee')
        .anchor('left')
        .add(pv.Label)
        .visible(function() { return this.index > 0 })
        .text(function(count) { return count } )
        .font('12px sans-serif')
        .parent
        .add(pv.Line)  // plot student counts
        .data(years)
        .left(x)
        .bottom(function(y) { return yStudents(summ.annual.getValue(this.index, summ.annualCol.students)) } )
        .add(pv.Dot)
        .fillStyle(function() { return this.strokeStyle() } )
        .title(function(y) { return String(y) + ': ' + summ.annual.getValue(this.index, summ.annualCol.students) + ' students tested' } )  // popup tip
        ;
    lh.anchor('top').add(pv.Label)  // left-hand title
        .top(-20)
        .text('Mount Greylock MCAS Performance')
        .font('bold 14px sans-serif')
        ;
    rh.anchor('top').add(pv.Label)  // right-hand title
        .top(-20)
        .text('Students Tested')
        .font('bold 14px sans-serif')
        ;
    vis.render()
        ;
}

function addPerformanceLevelsSeries(panel, x, h, normalize) {
    // Add to existing panel with horizontal year axis. x is the x scale function (year -> x position).
    // Display performance level frequency distributions over time.
    // Optionally normalize to 100% (relative frequency distributions).
    var years = summ.annual.getDistinctValues(summ.annualCol.year);
    if (normalize) {
        var yMax = 100;
        var yLabel = 'percentage of tests';
        function yValue(row, layer) { return summ.annual.getValue(row, summ.annualCol[layer]) / summ.annual.getValue(row, summ.annualCol.performance_level_count) * 100 }
        function yTotal(row) { return 100 }
    } else {
        var yMax = summ.annual.getColumnRange(summ.annualCol.performance_level_count).max;
        var yLabel = 'tests';
        function yValue(row, layer) { return summ.annual.getValue(row, summ.annualCol[layer]) }
        function yTotal(row) { return summ.annual.getValue(row, summ.annualCol.performance_level_count) }
    }
    var y = pv.Scale.linear(0, yMax).range(0, h);

    panel.add(pv.Layout.Stack)  // area plots
        .layers(['A','P','NI','WF'])
        .values(summ.annual.getSortedRows(0))  // row indices -- we don't really need to sort, but we need the array
        .x(function () { return x(summ.annual.getValue(this.index, summ.annualCol.year)) } )
        .y(function(row, layer) { return y(yValue(row, layer)) } )
        .layer.add(pv.Area)
        ;
    var legendLine = panel.anchor('right')
        .add(pv.Panel)
        .width(80)
        .height(5 * 20)
        .fillStyle('transparent')
        .add(pv.Bar)
        .data(['A','P','NI','WF', ''])
        .visible(function() { return this.index < 4 } )
        .fillStyle(function (layer) {return pv.Colors.category20(['A','P','NI','WF'])(layer)})
        .left(5)
        .width(20)
        .bottom(function () {return this.index * 20})
        .height(20)
        ;
    legendLine.add(pv.Dot)
        .visible(function() { return this.index == 4 } )
        .strokeStyle('#444')
        .fillStyle('#444')
        .bottom(function () {return this.index * 20 + 10})
        .left(15)
        .anchor('right')
        .add(pv.Label)
        .left(25)
        .text('CPI')
        .textAlign('left')
        ;
    legendLine.anchor('right')
        .add(pv.Label)
        .text(function (layer) {return '% ' + layer})
        .textAlign('left')
        ;
}

function displayAYP() {
    // Display AYP data.
    $('#AYP_view').empty();
    summarizeAnnual();
    var subjects = filterSelection(MCASSubjectCode);
    var aypForSubj = null;
    if (subjects && subjects.length == 1 && aypStateTarget[subjects[0]])
        aypForSubj = subjects[0];  // if just English or math is selected, include state AYP target line
    var pad = {left: 30, top: 40, right: 200, bottom: 20};  // pad around plots
    var w = 600, h = 400;
    var wFrame = w + pad.left + pad.right,
        hFrame = h + pad.top + pad.bottom;
    $('#AYP_view').width(wFrame).height(hFrame);
    var years = pv.range(2002, 2015);
    var cpiRange = summ.annual.getColumnRange(summ.annualCol.cpi);
    var cpiLow = Math.min(Math.floor(cpiRange.min / 10) * 10, 50);
    var nRows = summ.annual.getNumberOfRows();
    var cpi = new Object;
    for (var row = 0; row < nRows; row++)
        cpi[summ.annual.getValue(row, summ.annualCol.year)] = summ.annual.getValue(row, summ.annualCol.cpi);
    var safeHarbor = [false];
    for (var row = 1; row < nRows; row++) {
        var prevNumberNonprof = summ.annual.getValue(row - 1, summ.annualCol.NI) + summ.annual.getValue(row - 1, summ.annualCol.WF);
        var thisNumberNonprof = summ.annual.getValue(row, summ.annualCol.NI) + summ.annual.getValue(row, summ.annualCol.WF);
        safeHarbor[row] = thisNumberNonprof <= 0.9 * prevNumberNonprof;
    }
    var x = pv.Scale.ordinal(years).split(0, w);
    var xDelta = x(2014) - x(2013);
    var y = pv.Scale.linear(cpiLow, 100).range(0, h);
    var vis = new pv.Panel()  // root canvas
        .canvas('AYP_view')  // root canvas
        .width(w)
        .height(h)
        .left(pad.left)
        .right(pad.right)
        .top(pad.top)
        .bottom(pad.bottom)
        .fillStyle('#ccc')
        ;
    vis.add(pv.Rule)  // horizontal = years
        .data(years)
        .left(x)
        .strokeStyle('#eee')
        .anchor('bottom')
        .add(pv.Label)
        .text(function(y) { return y } )
        .font('12px sans-serif')
        ;
    vis.add(pv.Rule)  // vertical = CPI
        .data(y.ticks())
        .bottom(y)
        .width(10)
        .strokeStyle('#eee')
        .anchor('left')
        .add(pv.Label)
        .text(function(cpi) { return cpi } )
        .font('12px sans-serif')
        ;
    if (aypForSubj) {  // plot state AYP target line
        vis.add(pv.Line)
            .data(aypStateTarget.year)
            .left(x)
            .bottom(function() { return y(aypStateTarget[aypForSubj][this.index]) } )
            .strokeStyle('green')
            ;
    }
    vis.add(pv.Dot)  // plot NCLB target
        .data([2014])
        .strokeStyle('red')
        .left(x)
        .bottom(y(100))
        ;
    vis.add(pv.Line)  // plot CPI
        .data(pv.keys(cpi).sort())
        .strokeStyle('#444')
        .left(x)
        .bottom(function(yr) { return y(cpi[yr]) } )
        .add(pv.Dot)
        .fillStyle(function() { return this.strokeStyle() } )
        .title(function(yr) { return String(yr) + ' CPI: ' + summ.annual.getValue(this.index, summ.annualCol.cpi).toFixed() } )  // popup tip
        .add(pv.Dot)
        .visible(function() { return safeHarbor[this.index] } )
        .strokeStyle('lime')
        .fillStyle('transparent')
        .add(pv.Panel)  // plot annual targets
        .visible(function(yr) { return yr < 2014 } )
        .strokeStyle(null)
        .fillStyle(null)
        .add(pv.Line)
        .data(function(yr) { var gain = (y(100) - y(cpi[yr])) / (2014 - yr); return [[0, 0], [xDelta, gain]] } )
        .strokeStyle('blue')
        .left(function(d) { return d[0] } )
        .bottom(function(d) { return d[1] } )
        ;
    vis.anchor('right').add(pv.Panel)  // legend
        .height(5 * 20)
        .width(200)
        .fillStyle('transparent')
        .add(pv.Panel)  // legend row, from the bottom up
        .data(['state target line', 'safe harbor', 'annual target lines', 'NCLB goal', 'CPI'])
        .left(0)
        .bottom(function() { return this.index * 20 })
        .height(20)
        .add(pv.Panel)  // symbol
        .add(pv.Dot)
        .visible(function(caption) { return caption == 'CPI' || caption == 'NCLB goal' || caption == 'safe harbor' } )
        .left(10)
        .bottom(10)
        .strokeStyle(function(caption) { switch (caption) { 
                case 'CPI': return '#444'; 
                case'NCLB goal': return 'red'; 
                case 'safe harbor': return 'lime' } } )
        .fillStyle(function(caption) { switch (caption) {
                case 'CPI': return '#444'; 
                case'NCLB goal': return 'transparent'; 
                case 'safe harbor': return 'transparent' } } )
        .parent
        .add(pv.Line)  // annual target lines
        .data(function(caption) { return [[caption, 5], [caption, 15]] } )
        .visible(function(d) { return d[0] == 'annual target lines' || d[0] == 'state target line' && aypForSubj } )
        .strokeStyle(function(d) { return d[0] == 'annual target lines' ? 'blue' : 'green' } )
        .left(function(d) { return d[1] } )
        .bottom(10)
        .parent.parent
        .add(pv.Panel)  // caption
        .left(20)
        .add(pv.Label)
        .visible(function(caption) { return caption != 'state target line' || aypForSubj } )
        .textAlign('left')
        .font('12px sans-serif')
        .bottom(3)
        .text(function(caption) { return caption } )
        ;
    vis.render();
}

function displayGradesAndCohorts() {
    // Display performance over time for grade cross-sections and class cohorts.
    var element = $('#cohort_view');
    element.empty();
    summarizeClassGrade();
    // Assume summary is sorted by grade and year...
    var grades = summ.classGrade.getDistinctValues(summ.classGradeCol.grade);
    var gradeRange = { min: pv.min(grades), max: pv.max(grades) };
    var years =  summ.classGrade.getDistinctValues(summ.classGradeCol.year);
    var yearRange = { min: pv.min(years), max: pv.max(years) };
    var classes = summ.classGrade.getDistinctValues(summ.classGradeCol.classOf);
    var classRange = { min: pv.min(classes), max: pv.max(classes) };
    var wCanvas = screen.availWidth * 0.9; 
    // Grid.
    var wCell = Math.floor(wCanvas / ( yearRange.max - yearRange.min + 1 ) );  // partition width
    wCell = Math.min(wCell, 200);  // max 200 pixels
    wCanvas = wCell * ( yearRange.max - yearRange.min + 1 );
    // hCell = wCell
    var hCanvas = wCell * ( gradeRange.max - gradeRange.min + 1 );
    function scaleGrade(grade) { return wCell * ( grade - gradeRange.min ) }
    function scaleYear(year) { return wCell * ( year - yearRange.min ) }
    function scaleClass(classOf) { return wCell * ( classOf - classRange.min ) }
    var wGradeLabel = 0, hYearLabel = 20;
    var padding = {
         top: hYearLabel,
         bottom: 0,
         left: wGradeLabel,
         right: 0
    };
    element.width(wCanvas + padding.left + padding.right);
    element.height(hCanvas + padding.top + padding.bottom);
    function gradeData(grade) { 
        // Return a list of data for a particular grade, sorted by the class projected graduation year (classOf).
        var view = new google.visualization.DataView(summ.classGrade);
        view.setRows(summ.classGrade.getFilteredRows([{column: summ.classGradeCol.grade, value: grade}]));
        var sorted = view.getSortedRows(summ.classGradeCol.year);
        sorted = sorted.map(function(vRow) { return view.getTableRowIndex(vRow) } );
        view.setRows(sorted);
        var data = [];
        for (var row = 0; row < view.getNumberOfRows(); row++) {
            data.push( { year: view.getValue(row, summ.classGradeCol.year),
                    classOf: view.getValue(row, summ.classGradeCol.classOf),
                    total: view.getValue(row, summ.classGradeCol.performance_level_count),
                    A: view.getValue(row, summ.classGradeCol.A),
                    P: view.getValue(row, summ.classGradeCol.P),
                    NI: view.getValue(row, summ.classGradeCol.NI),
                    WF: view.getValue(row, summ.classGradeCol.WF)
                    } );
        }
        return data;
    }
    function classData(class) { 
        // Return a list of data for a particular class, sorted by grade.
        var view = new google.visualization.DataView(summ.classGrade);
        view.setRows(summ.classGrade.getFilteredRows([{column: summ.classGradeCol.classOf, value: class}]));
        var sorted = view.getSortedRows(summ.classGradeCol.grade);
        sorted = sorted.map(function(vRow) { return view.getTableRowIndex(vRow) } );
        view.setRows(sorted);
        var data = [];
        for (var row = 0; row < view.getNumberOfRows(); row++) {
            data.push( { 
                    grade: view.getValue(row, summ.classGradeCol.grade),
                    year: view.getValue(row, summ.classGradeCol.year),
                    total: view.getValue(row, summ.classGradeCol.performance_level_count),
                    A: {count: view.getValue(row, summ.classGradeCol.A)},
                    P: {count: view.getValue(row, summ.classGradeCol.P)},
                    NI: {count: view.getValue(row, summ.classGradeCol.NI)},
                    WF: {count: view.getValue(row, summ.classGradeCol.WF)}
                    } );
        }
        // Compute stacked starting positions in terms of test counts.
        for (var row = 0; row < data.length; row++) {
            data[row].WF.start = 0;
            data[row].NI.start = data[row].WF.start + data[row].WF.count;
            data[row].P.start = data[row].NI.start + data[row].NI.count;
            data[row].A.start = data[row].P.start + data[row].P.count;
        }
        return data;
    }
    var countMax = summ.classGrade.getColumnRange(summ.classGradeCol.performance_level_count).max;
    var scaleCount = pv.Scale.linear(0, countMax).range(0, 0.9 * wCell);
    var colorSeries = pv.Scale.ordinal('WF', 'NI', 'P', 'A').range(pv.color('rgba(255,0,0,0.7)'), pv.color('rgba(255,192,203,0.7)'), pv.color('rgba(0,0,255,0.7)'), pv.color('rgba(173,216,230,0.7)'));
    var vis = new pv.Panel()
        .canvas(element[0])
        .width(wCanvas)
        .height(hCanvas)
        .left(padding.left)
        .right(padding.right)
        .top(padding.top)
        .bottom(padding.bottom)
        ;
    // Label columns as years.
    vis.add(pv.Panel)
        .data(years)
        .width(wCell)
        .left(scaleYear)
        .anchor('top')
        .add(pv.Label)
        .text(function(year) { return year } )
        .top(- hYearLabel)
        .parent
        .add(pv.Rule)
        .left(wCell / 2)
        .strokeStyle('lightgray')
    // Draw gradewise performance history in horizontal panels.
    vis.add(pv.Panel)
        .data(grades)
        .height(wCell)
        .top(scaleGrade)
        .add(pv.Rule)
        .strokeStyle('lightgray')
        .bottom(0)
        .parent
        .anchor('left')
        .add(pv.Label)
        .text(function(grade) { return 'grade ' + grade } )
        .parent
        .add(pv.Layout.Stack)
        .offset('silohouette')
        .layers(['WF', 'NI', 'P', 'A'])
        .values(function(layer, grade) { return gradeData(grade) } )
        .x(function(d) { return scaleYear(d.year) + 0.5 * wCell } )
        .y(function(d, layer) { return scaleCount(d[layer]) } )
        .layer
        .add(pv.Area)
        .fillStyle(function(d, layer) { return colorSeries(layer) } )
        ;
    // Draw class performance history diagonally.
    vis.add(pv.Panel)
        .data(classes)
        .add(pv.Panel)
        .data(['WF', 'NI', 'P', 'A'])
        .add(pv.Area)
        .data(function(level, class) { return classData(class) } )
        .left(function(d) { return scaleYear(d.year) + 0.5 * wCell } )
        .bottom(function (d, level) { 
                var base = hCanvas - scaleGrade(d.grade) - wCell;
                console.log('grade ' + d.grade + ', base ' + base);
                var bot = base + (wCell - scaleCount(d.total)) / 2 + scaleCount(d[level].start);
                console.log('level ' + level + ', bottom ' + bot);
                return bot;
                } )
        .height(function(d, level) { return scaleCount(d[level].count) } )
        .fillStyle(function(d, level) { return colorSeries(level) } )
       /*
    vis.add(pv.Panel)
        .data(classes)
        .height(wCell)
        .top(topClass)
        .add(pv.Rule)
        .strokeStyle('lightgray')
        .top(0)
        .parent
        .anchor('left')
        .add(pv.Label)
        .text(function(class) { return 'class of ' + class } )
        //.left(- wClassLabel)
        .parent
        .add(pv.Layout.Stack)
        .offset('silohouette')
        .layers(['WF', 'NI', 'P', 'A'])
        .values(function(layer, class) { return classData(class) } )
        .x(function(d) { return leftGrade(d.grade) + 0.5 * wCell } )
        .y(function(d, layer) { return scaleCount(d[layer]) } )
        .layer
        .add(pv.Area)
        .fillStyle(function(d, layer) { return colorSeries(layer) } )
        ;
    // Add panel per grade/class (row of summary table).
    vis.add(pv.Panel)  .width(wCell).height(wCell)
        .data(pv.range(0, summ.classGrade.getNumberOfRows()))
        .left(function(r) { return leftGrade(summ.classGrade.getValue(r, summ.classGradeCol.grade)) } )
        .top(function(r) { return topClass(summ.classGrade.getValue(r, summ.classGradeCol.classOf)) } )
        .strokeStyle('orange')
        .anchor('left')
        .add(pv.Label)
        .text(function(r) { return r + ', year = ' +  summ.classGrade.getValue(r, summ.classGradeCol.year) + ', grade = ' + summ.classGrade.getValue(r, summ.classGradeCol.grade) })
        .textAlign('center')
        .textAngle(- Math.PI / 2 )
        .parent
        .add(pv.Bar)  // count of performance levels
        .data(function(r) { return [ summ.classGrade.getValue(r, summ.classGradeCol.performance_level_count) ] } )
        .width(0.2 * wCell)
        .left(0.4 * wCell)
        .height(scaleCount)
        .top(function(c) { return (wCell - scaleCount(c)) / 2 } )
        .fillStyle('gray')

        .add(pv.Layout.Stack)  // counts of each performance level
        .layers(function(total, r) { return ['WF', 'NI', 'P', 'A'].map(function(id) { return [ { id: id, count: summ.classGrade.getValue(r, summ.classGradeCol[id]) } ] } ) } )
        .y(function(d) { return scaleCount(d.count) } )
        .layer
        .add(pv.Bar)
        .fillStyle(function(d) { return colorSeries(d.id) } )
        .anchor('top')
        .add(pv.Label)
        .text(function(d) { return d.count } )
        ;
       */
    vis.render();
}

function displayPerformanceLevelsTrend(normalize) {
    // Display performance level frequency distributions over time.
    // Optionally normalize to 100% (relative frequency distributions).
    $("#primary_view_div").empty();
    summarizeAnnual();
    var w = 600, h = 300;
    var years = summ.annual.getDistinctValues(0);
    var x = pv.Scale.ordinal(years).split(0,w);
    if (normalize) {
        var yMax = 100;
        var yLabel = 'percentage of tests';
        function yValue(row) { return summ.annual.getValue(row, summ.annualCol[this.layer]) / summ.annual.getValue(row, summ.annualCol.performance_level_count) * 100 }
        function yTotal(row) { return 100 }
    } else {
        var yMax = summ.annual.getColumnRange(2).max;
        var yLabel = 'tests';
        function yValue(row) { return summ.annual.getValue(row, summ.annualCol[this.layer]) }
        function yTotal(row) { return summ.annual.getValue(row, summ.annualCol.performance_level_count) }
    }
    var y = pv.Scale.linear(0,yMax).range(0,h)

    var vis = new pv.Panel()
        .canvas('primary_view_div')
        .width(w)
        .height(h)
        .top(20)
        .bottom(20)
        .left(40)
        .right(80)
        ;
    vis.add(pv.Rule)  // vertical axis decoration
        .data(y.ticks())
        .left(0)
        .width(w)
        .bottom(y)
        .anchor('left')
        .add(pv.Label)
        .text(y.tickFormat)
        .root.anchor('left')
        .add(pv.Label)
        .text(yLabel)
        .left(-35)
        .textAngle(- Math.PI / 2)
        .textAlign('center')
        ;
    vis.add(pv.Layout.Stack)  // area plots
        .layers(['A','P','NI','WF'])
        .values(function(layer) {return summ.annual.getSortedRows(0).map(yValue, {layer: layer})})
        .x(function (yr) {return x(yr);})
        .y(function (val) {return y(val)})
        .layer.add(pv.Area);
    ;
    vis.anchor('right')  // legend
        .add(pv.Panel)
        .width(80)
        .height(5 * 20)
        .fillStyle('transparent')
        .add(pv.Bar)
        .data(['A','P','NI','WF', ''])
        .visible(function() { return this.index < 4 } )
        .fillStyle(function (layer) {return pv.Colors.category20(['A','P','NI','WF'])(layer)})
        .left(5)
        .width(20)
        .bottom(function () {return this.index * 20})
        .height(20)
        .add(pv.Dot)
        .visible(function() { return this.index == 4 } )
        .strokeStyle('#444')
        .fillStyle('#444')
        .bottom(function(d, b) { return b + 10 } )
        .anchor('right')
        .add(pv.Label)
        .text(function (layer) {return '% ' + layer})
        .textAlign('left')
        ;
    vis.add(pv.Rule)  // horizontal axis decoration
        .data(years)
        .left(function (yr) {return x(yr)})
        .bottom(0)
        .height(function () {return y(yTotal(this.index))})
        .strokeStyle('lightgray')
        .anchor('bottom')
        .add(pv.Label)
        .text(function (yr) {return yr})
        ;
    vis.render();
}

function displayCohortCrossSections(normalize) {
    // Display performance level frequency distributions over time,
    // cross-conditioned by grade and class.
    // Optionally normalize to 100% (relative frequency distributions).
    $("#primary_view_div").empty();
    summarizeClassGrade();
    var w = 900, h = 600;
    var classOf = summ.classGrade.getDistinctValues(0);
    var grades = summ.classGrade.getDistinctValues(1);
    // grid rows = classes, grid columns = grades
    cellSide = Math.min(h / classOf.length, w / grades.length);
    w = cellSide * grades.length;
    h = cellSide * classOf.length;
    var xGrid = pv.Scale.ordinal(grades).split(0, w);
    var yGrid = pv.Scale.ordinal(classOf).split(0, h);
    // Call the dimension within a cell z.
    if (normalize) {
        var zMax = 100;
        var zLabel = 'percentage of tests';
        function zValue(row) { return summ.classGrade.getValue(row, summ.classGradeCol[this.layer]) / summ.classGrade.getValue(row, summ.classGradeCol.performance_level_count) * 100 }
        function zTotal(row) { return 100 }
    } else {
        var zMax = summ.classGrade.getColumnRange(summ.classGradeCol.performance_level_count).max;
        var zLabel = 'tests';
        function zValue(row) { return summ.classGrade.getValue(row, summ.classGradeCol[this.layer]) }
        function zTotal(row) { return summ.classGrade.getValue(row, summ.classGradeCol.performance_level_count) }
    }
    // Scale z to leave room on the edges of cells.
    var z = pv.Scale.linear(0, zMax).range(0, 0.8 * cellSide);

    var vis = new pv.Panel()
        .canvas('primary_view_div')
        .width(w)
        .height(h)
        ;
    vis.add(pv.Panel)
        .data(classOf)
        .left(function (class) {return x(class) - cellSide / 2})
        .anchor('top');

        ;
}

</script>
    </script>
</head>
<body>
    <form>
        <table class='ui-widget'>
            <tr>
                <td>
                    <label title="Data files must contain tab-delimited text, with column headers.">
                        Data file:
                        <input type="file" name="input_data_file" size="100" onchange="newFile()">
                    </label>
                </td>
                <td><button type="button" id="open_data_file" onclick="readFile()" disabled>Open</button></td>
            </tr>
            <tr>
                <td class="partition" id='data_file_record_count'></td>
            </tr>
            <tr>
                <td class="partition"><div id="data_file_progress"></div></td>
            </tr>
        </table>
    </form>
    <table>
        <tr>
            <td>
                <form id="filter_control" style="visibility: hidden">
                    <table>
                        <tr>
                            <td id="filter_area">
                            </td>
                        </tr>
                        <tr>
                            <td>
                                <span id="filter_count"></span>
                                records selected
                            </td>
                        </tr>
                    </table>
                </form>
            </td>
        </tr>
        <tr><td id="metric_control">
    </table>
    <div id='view_control'>
        <ul>
            <li><a href="#column_view">variables</a></li>
            <li><a href="#table_view">table</a></li>
            <li><a href="#annual_summary_view">annual</a></li>
            <li><a href="#performance_view">performance</a></li>
            <li><a href="#AYP_view">AYP</a></li>
            <li><a href="#cohort_view">grades and cohorts</a></li>
        </ul>
        <div id="column_view">
        </div>
        <div id="table_view">
        </div>
        <div id="annual_summary_view">
        </div>
        <div id="performance_view">
        </div>
        <div id="AYP_view">
        </div>
        <div id="cohort_view">
        </div>
    </div>
</body>
</html>
